#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# 参考 :

import json
import openpyxl
import pandas as pd
import requests
from bs4 import BeautifulSoup
from openpyxl.utils.dataframe import dataframe_to_rows
import time

session = requests.session()

# ========================= ==========================

def update_session():
    # 这里模拟一下请求头，头文件是从浏览器里面抓到的，否则服务会回复403错误，（其实就是服务器做的简单防爬虫检测）
    headers = {
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0',
        'Content-Encoding': 'gzip',
        'Content-Type': 'text/html; charset=GBK',
        'Cookie': 'miid=1007934172757588344; cna=05NHFIYgRHYCAQ6D4/Nfb7wo; _tb_token_=TZ7OI0aChagqFW6VfOr3; cookie2=1a62bca767c1fc3c17065b463daf3917; v=0; dnk=wrfwsh; _samesite_flag_=true; thw=cn; tk_trace=oTRxOWSBNwn9dPyorMJE%2FoPdY8zMG1aAN%2F0TkjYGZjkj6rrK3kv4LgxGhtlxvv2n251iRZBLi5%2BxLVnyk1CiQFxLLAhdxWQkT%2BOCpmHPQRbPyN%2Bk8TAUjyMUjNsTljfXiMXo%2BN9Jn842NmgypZ3G7tv59ZgVBXg6nprznQ9HzDfqSPEDVELzdBgB%2BSuHg6qApf0q9x2%2BLtt8s3mG7PJS8DJgY8nJdloW9%2Bc7IS%2B%2BpYeLWjpA3Zu6SS8XWjXPIcO7nmYo7DvsWzR764RD8r4i7fTyF2DIYVbq4nDtF9iwNB4jYQDkiD35Ttc2A8TMNf8%2Bv7RwbMeXsyHBAn9zkAyXZs3eGuoN7WP4XpDqa2d6qKc1s9sMpT2Q%2FA%3D%3D; t=f22a3493240fe12c7d952c84aad09048; xlly_s=1; sgcookie=E100aHPlgV3odsHjbkzzOqmqWaQe4sO76p2j0EnV%2FvC%2Fs7u6htPnFW7XGcA5q9mTjActPRnArwXAdlvYERFXRd88Yg%3D%3D; unb=71965788; uc3=nk2=FOsxXsKU&lg2=Vq8l%2BKCLz3%2F65A%3D%3D&id2=VADRCwyvPqw%3D&vt3=F8dCufBEpDJHpwkR91c%3D; csg=37b540ab; lgc=wrfwsh; cookie17=VADRCwyvPqw%3D; skt=3321fb31c41b5e0e; existShop=MTYwMTUxMjc4MQ%3D%3D; uc4=nk4=0%40FmDlCHt2xu3MPuQx%2BKVJ1Fc%3D&id4=0%40Vhm8RJmhZJCCFhSGfsi4vVWnDQ%3D%3D; tracknick=wrfwsh; _cc_=VFC%2FuZ9ajQ%3D%3D; _l_g_=Ug%3D%3D; sg=h81; _nk_=wrfwsh; cookie1=WqIt8dJco6Cl0oYoCLBhWn5ErBK%2FS%2FapXLrhcO3N6Xo%3D; mt=ci=22_1; _m_h5_tk=8d3482c3d7b1ba864a477e989862548a_1601520413551; _m_h5_tk_enc=6ddb27fd2b9806dadcaada62f71d1132; uc1=cookie16=VT5L2FSpNgq6fDudInPRgavC%2BQ%3D%3D&cookie21=Vq8l%2BKCLjhZM&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D&existShop=false&pas=0&cookie14=Uoe0bHJlLWoLsQ%3D%3D&tag=8&lng=zh_CN; tfstk=cm6hBR_7zFgjxwRGGJ9Cc4mi3MVOaxceRTWdQOmUjA8PpPBBYsX-_j75IlYN5NG5.; l=eBxbZEjgQi7b5FCCBOfwhurza77OpKRAguPzaNbMiOCPO-CpQOtdWZz7H489CnGVh6X9R3Ss9LZQBeYBcfxWfdW2b_lHEhMmn; isg=BAsLX3c70KesXw3VV69qLxtdiqn1oB8iTREhnH0Ip8q1nCv-BXDtcrj2dpxyvHca'
    }
    session.headers.clear()
    session.headers.update(headers)

def get_page_by_url(city, page_url):  # 分析详细url获取所需信息
    strtime = time.strftime("%m-%d", time.localtime())
    # print("get_page_by_url:" + strtime + " " + page_url)
    update_session()
    try:
        res = session.get(page_url, timeout=(30, 120))

        # res = requests.get(page_url)
        if res.status_code == 200:
            soup = BeautifulSoup(res.text, "lxml")
            # print("soup:"+str(soup))

            for ul in soup.find_all('div', attrs={'class': 'sf-scroll-fixed'}):
                for li in ul.find_all('em', attrs={'class': 'count'}):
                    print(city + ":" + str(li.parent.text))
                    text = li.parent.text
                    return {city: text}
            return info
    except Exception as e:
        print(str(e))
    return None

def writer_to_text(list):  # 储存到text
    with open('阿里拍卖.text', 'a', encoding='utf-8')as f:
        f.write(json.dumps(list, ensure_ascii=False) + '\n')
        f.close()

def main(city, url):
    info = get_page_by_url(city, url)
    if info == "":
        return
    # writer_to_text(info)  # 储存到text文件
    time.sleep(2)

def do_write_workbook(ws, data):
    if data is None:
        return
    frame = pd.DataFrame(data, index=['0'])
    for r in dataframe_to_rows(frame, index=False, header=True):
        if '标题' in str(r):
            continue
        ws.append(r)

def create_sheet(wb, sheet_name):
    data = {'标题': '标题标题标题', '地址': 'url'}
    frame = pd.DataFrame(data, index=['0'])
    ws = wb.create_sheet(sheet_name)

    for r in dataframe_to_rows(frame, index=False, header=True):
        print('row:' + str(r))
        ws.append(r)
        break
    return ws

def has_sheet(wb, key):
    sheet_names = wb.get_sheet_names()
    for sheet_name in sheet_names:
        if key == sheet_name:
            return True
    return False

if __name__ == '__main__':
    dict = {
        '全国': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.23.685e3d88PqVfOg&category=50025969&auction_source=0&city=&province=&st_param=-1&auction_start_seg=-1',
        '北京': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.24.46b5533ai61Zd1&category=50025969&auction_source=0&city=&province=%B1%B1%BE%A9&st_param=-1&auction_start_seg=-1',
        '上海': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.34.7e4a7e7bjvA3c5&category=50025969&auction_source=0&city=&province=%C9%CF%BA%A3&st_param=-1&auction_start_seg=-1',
        '深圳': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.46.272244fdc3MUa8&category=50025969&auction_source=0&city=%C9%EE%DB%DA&province=&st_param=-1&auction_start_seg=-1',
        '广州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.44.351c6b82RvoT6e&category=50025969&auction_source=0&city=%B9%E3%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '天津': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.47.7cd83d88YHBRTy&category=50025969&auction_source=0&city=&province=%CC%EC%BD%F2&st_param=-1&auction_start_seg=-1',
        '杭州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.36.51714a75s1XR8y&category=50025969&auction_source=0&city=%BA%BC%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '南京': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.35.57cbfb3f4iNJZL&category=50025969&auction_source=0&city=%C4%CF%BE%A9&province=&st_param=-1&auction_start_seg=-1',
        '厦门': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.39.45632e12OGF1iM&category=50025969&auction_source=0&city=%CF%C3%C3%C5&province=&st_param=-1&auction_start_seg=-1',
        '成都': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.48.2af565fdiLNyKw&category=50025969&auction_source=0&city=%B3%C9%B6%BC&province=&st_param=-1&auction_start_seg=-1',
        '西安': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.52.e4aa3399DpyymV&category=50025969&auction_source=0&city=%CE%F7%B0%B2&province=&st_param=-1&auction_start_seg=-1',
        '重庆': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.47.74fa23cb9FQVZV&category=50025969&auction_source=0&city=&province=%D6%D8%C7%EC&st_param=-1&auction_start_seg=-1',
        '武汉': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.42.39c172229p3YMO&category=50025969&auction_source=0&city=%CE%E4%BA%BA&province=&st_param=-1&auction_start_seg=-1',
        '郑州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.41.3c1261a6S4Yprr&category=50025969&auction_source=0&city=%D6%A3%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '宁波': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.37.d7ff7db6P99nXR&category=50025969&auction_source=0&city=%C4%FE%B2%A8&province=&st_param=-1&auction_start_seg=-1',
        '温州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.38.6b33230bkd03My&category=50025969&auction_source=0&city=%CE%C2%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '无锡': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.36.46bdfb3f1cMb7T&category=50025969&auction_source=0&city=%CE%DE%CE%FD&province=&st_param=-1&auction_start_seg=-1',
        '常州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.38.6ba43415Vepvgx&category=50025969&auction_source=0&city=%B3%A3%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '苏州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.39.19395f4eYuduzi&category=50025969&auction_source=0&city=%CB%D5%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '佛山': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.49.1106799ciGzMiy&category=50025969&auction_source=0&city=%B7%F0%C9%BD&province=&st_param=-1&auction_start_seg=-1',
        '东莞': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.60.5ecc2f65yZFZPm&category=50025969&auction_source=0&city=%B6%AB%DD%B8&province=&st_param=-1&auction_start_seg=-1',
        '福州': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.38.6e50128aUrknNg&category=50025969&auction_source=0&city=%B8%A3%D6%DD&province=&st_param=-1&auction_start_seg=-1',
        '长沙': 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.43.30cd50d5LXWyWj&category=50025969&auction_source=0&city=%B3%A4%C9%B3&province=&st_param=-1&auction_start_seg=-1'
    }

    dict2 = {
        '全国': 'https://zc-paimai.taobao.com/list/0_______56950002.htm?spm=a219w.7474998.filter.19.46004615Oi40PJ&auction_source=0&st_param=-1&auction_start_seg=-1',
        '北京': 'https://zc-paimai.taobao.com/list/0____%B1%B1%BE%A9___56950002.htm?spm=a219w.7474998.filter.77.4c201b9cMKhs5t&auction_source=0&st_param=-1&auction_start_seg=-1',
        '上海': 'https://zc-paimai.taobao.com/list/0___%C9%CF%BA%A3____56950002.htm?spm=a219w.7474998.filter.30.41fc3c54VFBc9U&auction_source=0&st_param=-1&auction_start_seg=-1',
        '深圳': 'https://zc-paimai.taobao.com/list/0____%C9%EE%DB%DA___56950002.htm?spm=a219w.7474998.filter.42.351c56f02gzomb&auction_source=0&st_param=-1&auction_start_seg=-1',
        '广州': 'https://zc-paimai.taobao.com/list/0____%B9%E3%D6%DD___56950002.htm?spm=a219w.7474998.filter.40.57791041msH4kH&auction_source=0&st_param=-1&auction_start_seg=-1',
        '天津': 'https://zc-paimai.taobao.com/list/0___%CC%EC%BD%F2____56950002.htm?spm=a219w.7474998.filter.21.43116515yjeDHF&auction_source=0&st_param=-1&auction_start_seg=-1',
        '杭州': 'https://zc-paimai.taobao.com/list/0____%BA%BC%D6%DD___56950002.htm?spm=a219w.7474998.filter.32.721f2aadoIgb4a&auction_source=0&st_param=-1&auction_start_seg=-1',
        '南京': 'https://zc-paimai.taobao.com/list/0____%C4%CF%BE%A9___56950002.htm?spm=a219w.7474998.filter.31.3d26211dPvF7AY&auction_source=0&st_param=-1&auction_start_seg=-1',
        '厦门': 'https://zc-paimai.taobao.com/list/0____%CF%C3%C3%C5___56950002.htm?spm=a219w.7474998.filter.35.8d6c27f0se8cRD&auction_source=0&st_param=-1&auction_start_seg=-1',
        '成都': 'https://zc-paimai.taobao.com/list/0____%B3%C9%B6%BC___56950002.htm?spm=a219w.7474998.filter.44.36013badkZNRee&auction_source=0&st_param=-1&auction_start_seg=-1',
        '西安': 'https://zc-paimai.taobao.com/list/0____%CE%F7%B0%B2___56950002.htm?spm=a219w.7474998.filter.48.50cc797aqz5rWS&auction_source=0&st_param=-1&auction_start_seg=-1',
        '重庆': 'https://zc-paimai.taobao.com/list/0___%D6%D8%C7%EC____56950002.htm?spm=a219w.7474998.filter.41.50cc797a5IhtXL&auction_source=0&st_param=-1&auction_start_seg=-1',
        '武汉': 'https://zc-paimai.taobao.com/list/0____%CE%E4%BA%BA___56950002.htm?spm=a219w.7474998.filter.38.15d5c0aeuqnR3w&auction_source=0&st_param=-1&auction_start_seg=-1',
        '郑州': 'https://zc-paimai.taobao.com/list/0____%D6%A3%D6%DD___56950002.htm?spm=a219w.7474998.filter.37.39b06a4cUi7OY0&auction_source=0&st_param=-1&auction_start_seg=-1',
        '宁波': 'https://zc-paimai.taobao.com/list/0____%C4%FE%B2%A8___56950002.htm?spm=a219w.7474998.filter.33.40612aadjL4en7&auction_source=0&st_param=-1&auction_start_seg=-1',
        '温州': 'https://zc-paimai.taobao.com/list/0____%CE%C2%D6%DD___56950002.htm?spm=a219w.7474998.filter.34.75a7425ecRBbSE&auction_source=0&st_param=-1&auction_start_seg=-1',
        '无锡': 'https://zc-paimai.taobao.com/list/0____%CE%DE%CE%FD___56950002.htm?spm=a219w.7474998.filter.32.dea8211dNwALbr&auction_source=0&st_param=-1&auction_start_seg=-1',
        '常州': 'https://zc-paimai.taobao.com/list/0____%B3%A3%D6%DD___56950002.htm?spm=a219w.7474998.filter.34.785e30a7keQmpH&auction_source=0&st_param=-1&auction_start_seg=-1',
        '苏州': 'https://zc-paimai.taobao.com/list/0____%CB%D5%D6%DD___56950002.htm?spm=a219w.7474998.filter.35.785e30a7ocT8HR&auction_source=0&st_param=-1&auction_start_seg=-1',
        '佛山': 'https://zc-paimai.taobao.com/list/0____%B7%F0%C9%BD___56950002.htm?spm=a219w.7474998.filter.45.3b7756f04Yup6H&auction_source=0&st_param=-1&auction_start_seg=-1',
        '东莞': 'https://zc-paimai.taobao.com/list/0____%B6%AB%DD%B8___56950002.htm?spm=a219w.7474998.filter.56.104e252d60VF9W&auction_source=0&st_param=-1&auction_start_seg=-1',
        '福州': 'https://zc-paimai.taobao.com/list/0____%B8%A3%D6%DD___56950002.htm?spm=a219w.7474998.filter.34.48c44f914MJDz6&auction_source=0&st_param=-1&auction_start_seg=-1',
        '长沙': 'https://zc-paimai.taobao.com/list/0____%B3%A4%C9%B3___56950002.htm?spm=a219w.7474998.filter.39.d97e4121yUXIWr&auction_source=0&st_param=-1&auction_start_seg=-1'
    }
    for key, val in dict2.items():
        # print(key, val)
        main(key, val)








